{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "de765f16-341b-4dc3-b59d-b16437c7e050",
   "metadata": {},
   "source": [
    "\n",
    "# RAG with structurizr sample codes downloaded from github\n",
    "\n",
    "### 1. Example structurizr code\n",
    "\n",
    "Please refer to the content of dsl code below in this demo.\n",
    "\n",
    "[examples/dsl/big-bank-plc/model.dsl](examples/dsl/big-bank-plc/model.dsl)\n",
    "\n",
    "The code is clone from `https://github.com/structurizr/examples.git`\n",
    "\n",
    "### 2. structurizr language reference\n",
    "\n",
    "[workspace/structurizr/dsl/docs/language-reference.md](workspace/structurizr/dsl/docs/language-reference.md)\n",
    "\n",
    "The code is clone from `https://github.com/structurizr/dsl.git`\n",
    "\n",
    "Please refer to `README-MarkdownLoader.md` for environment setup."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e25cdbdc-daee-4a9b-b363-9080dc216927",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Name: langchain\n",
      "Version: 0.1.9\n",
      "Summary: Building applications with LLMs through composability\n",
      "Home-page: https://github.com/langchain-ai/langchain\n",
      "Author: \n",
      "Author-email: \n",
      "License: MIT\n",
      "Location: /opt/conda/lib/python3.11/site-packages\n",
      "Requires: aiohttp, dataclasses-json, jsonpatch, langchain-community, langchain-core, langsmith, numpy, pydantic, PyYAML, requests, SQLAlchemy, tenacity\n",
      "Required-by: \n"
     ]
    }
   ],
   "source": [
    "!pip show langchain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e07b2c7d-62d7-457c-bbcb-298b29aa522c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Please define the following env variables in the .env file \n",
    "# \n",
    "# ollama_url = \"http://ollama:11434\"\n",
    "# model_name = \"mistral:instruct\"\n",
    "# "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "fa2048ea-dbc4-49d7-b855-8b546dce2edb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use the utility.py to get llm, embedding and output parser\n",
    "import utility\n",
    "embeddings = utility.get_embeddings()\n",
    "llm = utility.get_llm()\n",
    "output_parser = utility.get_output_parser()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a11baec0-71af-46f8-bb39-bacc1c354c55",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1/1 [00:00<00:00, 307.66it/s]\n"
     ]
    }
   ],
   "source": [
    "# Load the dsl file from local machine for embedding\n",
    "# https://python.langchain.com/docs/modules/data_connection/document_loaders/file_directory\n",
    "# https://python.langchain.com/docs/integrations/document_loaders/unstructured_file\n",
    "\n",
    "from langchain_community.document_loaders import DirectoryLoader, TextLoader\n",
    "\n",
    "# only load examples/dsl/big-bank-plc/model.dsl\n",
    "loader = DirectoryLoader('examples/dsl/big-bank-plc', glob=\"model.dsl\", show_progress=True, use_multithreading=True, loader_cls=TextLoader)\n",
    "\n",
    "docs1 = loader.load()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "97c4e0ee-d1b7-4f11-914c-4b19d7812b06",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Document(page_content='workspace {\\n\\n    model {\\n        customer = person \"Personal Banking Customer\" \"A customer of the bank, with personal bank accounts.\" \"Customer\"\\n\\n        group \"Big Bank plc\" {\\n            supportStaff = person \"Customer Service Staff\" \"Customer service staff within the bank.\" \"Bank Staff\"\\n            backoffice = person \"Back Office Staff\" \"Administration and support staff within the bank.\" \"Bank Staff\"\\n\\n            mainframe = softwaresystem \"Mainframe Banking System\" \"Stores all of the core banking information about customers, accounts, transactions, etc.\" \"Existing System\"\\n            email = softwaresystem \"E-mail System\" \"The internal Microsoft Exchange e-mail system.\" \"Existing System\"\\n            atm = softwaresystem \"ATM\" \"Allows customers to withdraw cash.\" \"Existing System\"\\n\\n            internetBankingSystem = softwaresystem \"Internet Banking System\" \"Allows customers to view information about their bank accounts, and make payments.\"\\n        }\\n\\n        # relationships between people and software systems\\n        customer -> internetBankingSystem \"Views account balances, and makes payments using\"\\n        internetBankingSystem -> mainframe \"Gets account information from, and makes payments using\"\\n        internetBankingSystem -> email \"Sends e-mail using\"\\n        email -> customer \"Sends e-mails to\"\\n        customer -> supportStaff \"Asks questions to\" \"Telephone\"\\n        supportStaff -> mainframe \"Uses\"\\n        customer -> atm \"Withdraws cash using\"\\n        atm -> mainframe \"Uses\"\\n        backoffice -> mainframe \"Uses\"\\n    }\\n\\n}', metadata={'source': 'examples/dsl/big-bank-plc/model.dsl'})]\n"
     ]
    }
   ],
   "source": [
    "# print the loaded documents, the model.dsl content has been loaded as the page_content\n",
    "print(docs1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "73852a42-1f36-4de9-8737-c68bb9def2ea",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Created a chunk of size 261, which is longer than the specified 200\n",
      "Created a chunk of size 386, which is longer than the specified 200\n",
      "Created a chunk of size 588, which is longer than the specified 200\n"
     ]
    }
   ],
   "source": [
    "# Split the document content into small chunks of texts for embedding\n",
    "\n",
    "from langchain.text_splitter import CharacterTextSplitter\n",
    "text_splitter1 = CharacterTextSplitter(chunk_size=200, chunk_overlap=50)\n",
    "texts1 = text_splitter1.split_documents(docs1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "ab3296e7-e2ab-4026-bd41-49d6b0b25230",
   "metadata": {},
   "outputs": [],
   "source": [
    "# only load structurizr/dsl/docs/language-reference.md\n",
    "# https://python.langchain.com/docs/modules/data_connection/document_loaders/markdown\n",
    "# pip install unstructured\n",
    "from langchain_community.document_loaders import UnstructuredMarkdownLoader\n",
    "mdloader = UnstructuredMarkdownLoader(\"structurizr/dsl/docs/language-reference.md\")\n",
    "\n",
    "docs2 = mdloader.load()\n",
    "\n",
    "from langchain.text_splitter import MarkdownTextSplitter\n",
    "\n",
    "text_splitter2 = MarkdownTextSplitter(chunk_size=200, chunk_overlap=50)\n",
    "\n",
    "texts2 = text_splitter2.split_documents(docs2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "fbb96656-7c7f-4896-a12f-ebe589e0ce3c",
   "metadata": {},
   "outputs": [],
   "source": [
    "texts = texts1 + texts2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "93f03020-3887-4aae-9ece-6bd088d51d49",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "from langchain_community.vectorstores import DocArrayInMemorySearch\n",
    "\n",
    "vectorstore = DocArrayInMemorySearch.from_documents(texts, embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "02b3f72b-a7e1-48f9-8a08-e61399644775",
   "metadata": {},
   "outputs": [],
   "source": [
    "retriever = vectorstore.as_retriever()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "081c8e88-e45c-4b32-bc0e-206801565f83",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_core.runnables import RunnableParallel, RunnablePassthrough\n",
    "\n",
    "retrieval = RunnableParallel(\n",
    "    {\"context\": retriever, \"input\": RunnablePassthrough()}\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "dd527cf0-379d-4800-bff7-02b7cfdf41c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_core.prompts import ChatPromptTemplate\n",
    "\n",
    "prompt = ChatPromptTemplate.from_messages([\n",
    "    (\"system\", \"You are a software architect with knowledage in structurizr helping software architecture modeling and documenation in the following context: {context}\"),\n",
    "    (\"user\", \"{input}\")\n",
    "])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "fd5a6f70-6f06-4360-b35f-8a5509c36bc8",
   "metadata": {},
   "outputs": [],
   "source": [
    "chain = retrieval | prompt | llm | output_parser"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "336770a5-a375-4199-9759-bf1753ff774a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Based on the context provided, an \"Internet Banking System\" can be modeled using Structurizr as follows:\n",
      "\n",
      "1. **System Context View**: This view includes the Internet Banking System in scope, along with all people and software systems that are directly connected to it. For example, this could include users accessing the system via a web browser, an authentication server, and possibly other integrations with external financial institutions.\n",
      "\n",
      "2. **Container Views**: Each major component of the Internet Banking System can be modeled as a separate container view. For instance, you might have containers for the User Interface (UI), Authentication and Authorization, Transaction Processing, Database Access, and so on. Containers represent the boundaries of each component and show all other directly connected components or systems.\n",
      "\n",
      "3. **Naming Conventions**: Make sure to define clear and consistent names for your system, containers, and their connections. In this example, you could name the Internet Banking System as \"InternetBankingSystem\", and container names like \"UIContainer_InternetBankingSystem\" or \"AuthAndAuthorization_InternetBankingSystem\".\n",
      "\n",
      "4. **Extending the Workspace**: As your model evolves, you may choose to add more features or components, which can be added as new containers or systems in the extended workspace while using the identifiers defined earlier. This helps maintain a consistent and well-organized architecture model.\n"
     ]
    }
   ],
   "source": [
    "result = chain.invoke(\"\"\"What is the \"Internet Banking System\" \"\"\")\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "b2ec70bc-9810-4af9-af64-0a507958cc83",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Based on the context provided, an \"E-mail System\" can be modeled using different views in Structurizr depending on the level of detail and scope required. Here are some possible ways to represent an E-mail System:\n",
      "\n",
      "1. Component View: In this view, the E-mail System would be represented as a single component that includes all the sub-components and elements responsible for handling e-mail functions like email sender, recipient manager, message composition, transport protocols (SMTP, POP3, IMAP), authentication, encryption, etc.\n",
      "\n",
      "2. System Context View: This view expands the scope to include not only the E-mail System but also all software systems and people that are directly connected to it. For instance, you might have an external mail server or a user interface for composing emails.\n",
      "\n",
      "3. Container View: In this case, you would represent the E-mail System as a container encompassing all containers within the software system responsible for email processing. This could include containers such as a Mail Transfer Agent (MTA), Mail User Agent (MUA), and other related components.\n",
      "\n",
      "In summary, the \"E-mail System\" can be modeled in various ways depending on your desired level of detail and scope using the Component View, System Context View, or Container View in Structurizr."
     ]
    }
   ],
   "source": [
    "# print(chain.invoke(\"\"\"What is the \"E-mail System\" \"\"\"))\n",
    "for chunk in chain.stream(\"\"\"What is the \"E-mail System\" \"\"\"):\n",
    "    print(chunk, end=\"\", flush=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "23a2835a-bd0e-42ad-af42-8b05afb09906",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " I'm sorry for any confusion, but it seems there is a misunderstanding in the context of your question. The provided documents discuss software architecture modeling and documentation using the Structurizr DSL language. There is no mention or relevance to bank staff in the context given. If you have any specific questions related to the contents of those documents, please let me know.\n"
     ]
    }
   ],
   "source": [
    "# sample to run a batch of queries\n",
    "for result in chain.batch([\"\"\"Who are the bank staff \"\"\"]):\n",
    "    print(result,flush=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "09df3b1e-40ee-4f2e-926b-9c722395328a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Based on your requirements, here is an example of how you could model this in Structurizr DSL:\n",
      "\n",
      "```dsl\n",
      "person bigBankPlc {\n",
      "  name \"Big Bank plc\"\n",
      "  tags [\"Bank\", \"Corporate\"]\n",
      "}\n",
      "\n",
      "softwareSystem atm {\n",
      "  name \"ATM System\"\n",
      "  description \"Application used by customers to withdraw cash\"\n",
      "  containedBy bigBankPlc\n",
      "}\n",
      "\n",
      "softwareSystem mainframe {\n",
      "  name \"Mainframe System\"\n",
      "  description \"Application used by bank staff to search customer information\"\n",
      "  containedBy bigBankPlc\n",
      "}\n",
      "\n",
      "impliedRelationship {\n",
      "  from atm\n",
      "  to mainframe\n",
      "  relationshipType \"Data Access\"\n",
      "}\n",
      "```\n",
      "\n",
      "This code defines a `person` with the name `Big Bank plc`, and two `softwareSystems`: `ATM System` and `Mainframe System`. The `ATM System` is contained by `Big Bank plc` as it will be used by customers, while the `Mainframe System` is also contained by `Big Bank plc` as it will be used by bank staff. An `impliedRelationship` is defined between these two systems to represent the data access relationship between them."
     ]
    }
   ],
   "source": [
    "# requirements = \"\"\"The \"Big Bank plc\" wants to create a new application for its bank staff and customers, \n",
    "# the customers will use \"ATM\" system to withdraw cash, the bank staff will use mainframe system to search customer information.\n",
    "# \"\"\"\n",
    "\n",
    "instruct = \"\"\" Please generate the structurizr DSL codes according to user requirements: \n",
    "1. the \"Big Bank plc\" wants to create a new application for its bank staff and customers, \n",
    "2. the customers will use \"ATM\" system to withdraw cash, the bank staff will use mainframe system to search customer information.\n",
    "\"\"\"\n",
    "\n",
    "for chunk in chain.stream(instruct):\n",
    "    print(chunk, end=\"\", flush=True)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "40b96bb3-766a-4c6d-bd51-da60d283f77d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Based on your requirements, here is the Structurizr DSL code for the \"Big Bank plc\" application:\n",
      "\n",
      "```dsl\n",
      "workspace \"Big Bank plc Application\" {\n",
      "    description \"A new application for bank staff and customers of Big Bank plc\"\n",
      "\n",
      "    model {\n",
      "        //define the persons here\n",
      "        user = person \"Customer\"\n",
      "        bankStaff = person \"Bank Staff\"\n",
      "\n",
      "        //define the software systems here, it can have multiple systems\n",
      "        atmSystem = softwareSystem \"ATM System\" {\n",
      "            description \"The system used by customers to withdraw cash\"\n",
      "            technology \"Java\"\n",
      "            tags {\"hardware\", \"banking\"}\n",
      "        }\n",
      "\n",
      "        mainframeSystem = softwareSystem \"Mainframe System\" {\n",
      "            description \"The system used by bank staff to search customer information\"\n",
      "            technology \"COBOL\"\n",
      "            tags {\"legacy\", \"banking\"}\n",
      "        }\n",
      "\n",
      "        //define the relationship here\n",
      "        user -> atmSystem \"Uses\"\n",
      "        atmSystem -> database \"Reads from and writes to\"\n",
      "        bankStaff -> mainframeSystem \"Uses\"\n",
      "    }\n",
      "\n",
      "    views {\n",
      "        systemLandscape {\n",
      "            include softwareSystem.atmSystem, softwareSystem.mainframeSystem\n",
      "            autolayout lr\n",
      "        }\n",
      "    }\n",
      "}\n",
      "```\n",
      "\n",
      "In this code snippet, we define the `workspace` for the \"Big Bank plc Application\" with its description. Then, within the `model`, we define the persons (customer and bank staff), software systems (ATM System and Mainframe System), and their relationships as specified in your requirements. Finally, we include both software systems in the `systemLandscape` view using the `include` statement and set the layout to left-to-right using `autolayout lr`."
     ]
    }
   ],
   "source": [
    "instruct = \"\"\" Please generate the structurizr DSL codes according to user requirements: \n",
    "1. the \"Big Bank plc\" wants to create a new application for its bank staff and customers, \n",
    "2. the customers will use \"ATM\" system to withdraw cash, the bank staff will use mainframe system to search customer information.\n",
    "3. The structurizr DSL codes should follow the standards defined in language-reference.md\n",
    "4. The structurizr DSL codes should follow the following template:\n",
    "\n",
    "workspace {\n",
    "    name \"placehoder for application name\"\n",
    "    descritption \"placehoder for application descritption\"\n",
    "\n",
    "    model {\n",
    "        //define the persons here\n",
    "        user = person \"User\"\n",
    "        \n",
    "        //define the software systems here, it can have multiple systems\n",
    "        softwareSystem = softwareSystem \"Software System\" \n",
    "        \n",
    "        //define the relationship here\n",
    "        user -> webapp \"Uses\"\n",
    "        webapp -> database \"Reads from and writes to\"\n",
    "        \n",
    "    }\n",
    "\n",
    "    //do not change this section\n",
    "    views {\n",
    "        systemLandscape softwareSystem {\n",
    "            include *\n",
    "            autolayout lr\n",
    "        }\n",
    "    }\n",
    "\n",
    "}\"\"\"\n",
    "\n",
    "for chunk in chain.stream(instruct):\n",
    "    print(chunk, end=\"\", flush=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "61e67eb8-bb5a-4ce0-89bf-3b4ab072333d",
   "metadata": {},
   "source": [
    "\n",
    "## Notes\n",
    "\n",
    "Open https://structurizr.com/dsl and put the above dsl codes into the online editor to see the architecture diagram.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "43274f00-5f98-4658-838b-ab8d65c7a071",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Based on the user requirements and the provided template, here's the Structurizr DSL code:\n",
      "\n",
      "```dsl\n",
      "workspace \"Big Bank plc Application\" {\n",
      "    description \"A new application for bank staff and customers of Big Bank plc.\"\n",
      "\n",
      "    model {\n",
      "        //define persons\n",
      "        user = person \"Customer\"\n",
      "        bankStaff = person \"Bank Staff\"\n",
      "\n",
      "        //define software systems\n",
      "        atmSystem = softwareSystem \"ATM System\" {\n",
      "            description \"The system used by customers to withdraw cash.\"\n",
      "            technology \"Java, Spring Boot\"\n",
      "            tags {\"Hardware\", \"Banking\"}\n",
      "        }\n",
      "        mainframeSystem = softwareSystem \"Mainframe System\" {\n",
      "            description \"The system used by bank staff to search customer information.\"\n",
      "            technology \"COBOL, IBM z/OS\"\n",
      "            tags {\"Legacy\", \"Banking\"}\n",
      "        }\n",
      "\n",
      "        //define relationships\n",
      "        user -> atmSystem \"Uses\"\n",
      "        bankStaff -> mainframeSystem \"Uses\"\n",
      "        atmSystem -> database \"Reads from and writes to\"\n",
      "    }\n",
      "\n",
      "    views {\n",
      "        systemLandscape {\n",
      "            include *\n",
      "            autolayout lr\n",
      "        }\n",
      "    }\n",
      "}\n",
      "\n",
      "// Extend the workspace with any additional elements, relationships, or views if needed\n",
      "workspace extends \"./path/to/extended/workspace.stg\" {\n",
      "    // Add any extra definitions here\n",
      "}\n",
      "```\n",
      "\n",
      "Make sure to replace `./path/to/extended/workspace.stg` with the actual path to your extended workspace file (if applicable)."
     ]
    }
   ],
   "source": [
    "instruct = \"\"\" Please generate the structurizr DSL codes according to user requirements: \n",
    "1. the \"Big Bank plc\" wants to create a new application for its bank staff and customers, \n",
    "2. the customers will use \"ATM\" system to withdraw cash, the bank staff will use mainframe system to search customer information.\n",
    "3. The structurizr DSL codes should follow the standards defined in language-reference.md\n",
    "4. The structurizr DSL codes should follow the following template:\n",
    "\n",
    "workspace {\n",
    "    name \"placehoder for application name\"\n",
    "    descritption \"placehoder for application descritption\"\n",
    "\n",
    "    model {\n",
    "        //define the persons here\n",
    "        user = person \"User\"\n",
    "        \n",
    "        //define the software systems here, it can have multiple systems\n",
    "        softwareSystem = softwareSystem \"Software System\" \n",
    "        \n",
    "        //define the relationship here\n",
    "        user -> webapp \"Uses\"\n",
    "        webapp -> database \"Reads from and writes to\"\n",
    "        \n",
    "    }\n",
    "\n",
    "    //do not change this section\n",
    "    views {\n",
    "        systemLandscape softwareSystem {\n",
    "            include *\n",
    "            autolayout lr\n",
    "        }\n",
    "    }\n",
    "\n",
    "}\"\"\"\n",
    "\n",
    "for chunk in chain.stream(instruct):\n",
    "    print(chunk, end=\"\", flush=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a9f922b8-2fb5-417f-8f2f-2475e48b482d",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
